from bs4 import BeautifulSoup
import requests
import time
from collections import Counter

'''
https://www.tripadvisor.cn/Attractions-g60763-Activities-c47-New_York_City_New_York.html#FILTERED_LIST
https://www.tripadvisor.cn/Attractions-g60763-Activities-c47-oa30-New_York_City_New_York.html#FILTERED_LIST
https://www.tripadvisor.cn/Attractions-g60763-Activities-c47-oa60-New_York_City_New_York.html#FILTERED_LIST
'''

url = 'https://www.tripadvisor.cn/Attractions-g60763-Activities-New_York_City_New_York.html'
url_profile = 'https://www.tripadvisor.cn/MemberProfile-a_uid.BC47ABD04CBD509833E79CE53D6CCA1D'
urls = ['https://www.tripadvisor.cn/Attractions-g60763-Activities-c47-oa{}-New_York_City_New_York.html#FILTERED_LIST'.format(str(i)) for i in range(0, 30*15, 30)]

def get_attractionss(url):
    web_data= requests.get(url)
    soup = BeautifulSoup(web_data.text,'lxml')
    titles = soup.select("#ATTR_ENTRY_ > div.attraction_clarity_cell > div > div > div.listing_info > div.listing_title > a[target='_blank']")
    imgs = soup.select("img[width='180']")
    tags = soup.select("#ATTR_ENTRY_ > div.attraction_clarity_cell > div > div > div.listing_info > div.tag_line > div")

    data = []
    for title, img, tag in zip(titles, imgs, tags):
        temp = list(tag.stripped_strings)
        info = {
            "title": title.get_text(),
            "img": img.get('src'),
            "tag": [i for i in temp if i != ',']
        }
        data.append(info)
    # print(data)
    return data

datas = []

for temp in urls:
    time.sleep(2)
    data = get_attractionss(temp)
    # print(data)
    datas.extend(data)

tags = []
for data in datas:
    tags.extend(data.get("tag"))
c = Counter(tags)
print(c.most_common(3))

'''
web_data = requests.get(url)
soup = BeautifulSoup(web_data.text,'lxml')

titles = soup.select("#ATTR_ENTRY_ > div.attraction_clarity_cell > div > div > div.listing_info > div.listing_title > a[target='_blank']")
imgs = soup.select("img[width='180']")
tags = soup.select("#ATTR_ENTRY_ > div.attraction_clarity_cell > div > div > div.listing_info > div.tag_line > div")

for title, img, tag in zip(titles, imgs, tags):
    temp = list(tag.stripped_strings)
    data = {
        "title": title.get_text(),
        "img": img.get('src'),
        "tag": [i for i in temp if i != ',']
    }
    print(data)
'''
'''

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36',
    'Cookie': 'TAUnique=%1%enc%3A0K2Wy69nxlH4bI3bW6BVnhDm0Igrj%2BVTMTAYdjYI%2BBXiJ6g91gwMNg%3D%3D; _ga=GA1.2.1706238189.1515580016; TAPD=tripadvisor.cn; _ym_uid=151558001758585750; __gads=ID=ea07be7dafc6a4c2:T=1515580023:S=ALNI_MYbY2BLRSWDPAPiVa4jrfVb6xcpDg; ki_r=; CM=%1%HanaPersist%2C%2C-1%7CPremiumMobSess%2C%2C-1%7Ct4b-pc%2C%2C-1%7CHanaSession%2C%2C-1%7CRestAds%2FRPers%2C%2C-1%7CRCPers%2C%2C-1%7CWShadeSeen%2C%2C-1%7CFtrPers%2C%2C-1%7CTheForkMCCPers%2C%2C-1%7CHomeASess%2C%2C-1%7CPremiumSURPers%2C%2C-1%7CPremiumMCSess%2C%2C-1%7CRestPartSess%2C%2C-1%7CRestPremRSess%2C%2C-1%7CCCSess%2C%2C-1%7CPremRetPers%2C%2C-1%7CViatorMCPers%2C%2C-1%7Csesssticker%2C%2C-1%7CPremiumORSess%2C%2C-1%7Ct4b-sc%2C%2C-1%7CRestAdsPers%2C%2C-1%7CMC_IB_UPSELL_IB_LOGOS2%2C%2C-1%7Cb2bmcpers%2C%2C-1%7CMC_IB_UPSELL_IB_LOGOS%2C%2C-1%7CPremMCBtmSess%2C%2C-1%7CPremiumSURSess%2C%2C-1%7CLaFourchette+Banners%2C%2C-1%7Csess_rev%2C%2C-1%7Csessamex%2C%2C-1%7CPremiumRRSess%2C%2C-1%7CSaveFtrPers%2C%2C-1%7CSPMCSess%2C%2C-1%7CTheForkORSess%2C%2C-1%7CTheForkRRSess%2C%2C-1%7Cpers_rev%2C%2C-1%7CMetaFtrSess%2C%2C-1%7CRBAPers%2C%2C-1%7CWAR_RESTAURANT_FOOTER_PERSISTANT%2C%2C-1%7CFtrSess%2C%2C-1%7CRestAds%2FRSess%2C%2C-1%7CHomeAPers%2C%2C-1%7CPremiumMobPers%2C%2C-1%7CRCSess%2C%2C-1%7CLaFourchette+MC+Banners%2C%2C-1%7CRestAdsCCSess%2C%2C-1%7CRestPartPers%2C%2C-1%7CRestPremRPers%2C%2C-1%7Csh%2C%2C-1%7Cpssamex%2C%2C-1%7CTheForkMCCSess%2C%2C-1%7CCCPers%2C%2C-1%7CWAR_RESTAURANT_FOOTER_SESSION%2C%2C-1%7Cb2bmcsess%2C%2C-1%7CSPMCPers%2C%2C-1%7CPremRetSess%2C%2C-1%7CViatorMCSess%2C%2C-1%7CPremiumMCPers%2C%2C-1%7CPremiumRRPers%2C%2C-1%7CRestAdsCCPers%2C%2C-1%7CTheForkORPers%2C%2C-1%7CPremMCBtmPers%2C%2C-1%7CTheForkRRPers%2C%2C-1%7CSaveFtrSess%2C%2C-1%7CPremiumORPers%2C%2C-1%7CRestAdsSess%2C%2C-1%7CRBASess%2C%2C-1%7CSPORPers%2C%2C-1%7Cperssticker%2C%2C-1%7CMetaFtrPers%2C%2C-1%7C; ServerPool=X; TASSK=enc%3AAIhL2zDhJA4VFDe6L1a9u9AALNdXFh6lSfGEtReE%2BzRs%2BG4jjq4yuAPT%2B0RbEx8fmEPwAI0rH4A%2BSGj8wkyw8eFQ%2FG2H3siW2ubSnXXOHj2MtXcO9y%2BLM1OuQvNd3sjoIg%3D%3D; _gid=GA1.2.49814982.1516252327; _ym_isad=2; SecureLogin2=3.4%3AADZ4HE7YFnib3gMNua6CjqY6pbqgy4v1YlF1tXB7uEBedPBAyg3IkqavQDEKNTBagDiWDoW77w9TEKnHudzVJzgAOpbX2%2BbwtcLa9%2FnU7YNRG4dl3cg%2FttN9ZX5lNoBlkkEH%2FE%2FHXyBVyPt%2FNeREqZwOmp%2Fo8H7NiFmMCs6aAJyJ%2FgRbGstnGlG2A0guEli4%2B7GJtSHTp%2FkFawW4%2By2Vpqc%3D; TAAuth3=3%3A0e153c4043c566c4b9cd6d13816334e0%3AAJsmL4%2Ff3FMMNi8OfUOMy%2Fgp33StyxV1z2ek%2FOp9ZC%2FUQCTaFW7JVUIrpasX2B8b%2BwiQDYeGMpd7g6RlSKFp0YnF6yJamIehGOshgDZow4yccHJcDSfaaqV99QYa5a%2FIMoehi61FIpHlhFT6HjvQ0ZIn5q%2Fny0mPp1zlP81mR3pJtLf%2FYXxj5AbbtaDReN6ypg%3D%3D; BEPIN=%1%16107afde64%3Busr04t.daodao.com%3A10023%3B; CommercePopunder=SuppressAll*1516252429170; TAReturnTo=%1%%2FAttractions-g60763-Activities-New_York_City_New_York.html; roybatty=TNI1625!AEm1vU4jaWlnirj0otLujmVuwHM5cZTvSvSJL%2FZdqmBRTpy2N8F5sdxWW5g1J4aGuoaatH%2FTjg7QX3vPPZv%2FzhXmn%2FffgLOSkZY8uVcbzihqXXvQV3ohT75tXYOONujkiLRKs7ldqF9H6DiqSjj17D1rlFyNOHHwr5lV1%2Fv5kTiO%2C1; ki_t=1515580023800%3B1516252402358%3B1516253422891%3B5%3B38; TASession=%1%V2ID.654B378F9F062E024030EC318EB075FE*SQ.140*LP.%2FAttractions-g60763-Activities-New_York_City_New_York%5C.html*PR.427%7C*LS.DemandLoadAjax*GR.66*TCPAR.15*TBR.88*EXEX.44*ABTR.16*PHTB.22*FS.5*CPU.7*HS.recommended*ES.popularity*AS.popularity*DS.5*SAS.popularity*FPS.oldFirst*TS.BC47ABD04CBD509833E79CE53D6CCA1D*LF.zhCN*FA.1*DF.0*IR.3*OD.en_US*MS.-1*RMS.-1*FLO.60763*TRA.true*LD.60763; TATravelInfo=V2*AY.2018*AM.1*AD.28*DY.2018*DM.1*DD.29*A.2*MG.-1*HP.2*FL.3*DSM.1516253465305*RS.1; TAUD=LA-1516252327717-1*RDD-1-2018_01_18*HDD-93561-2018_01_28.2018_01_29*LD-1137566-2018.1.28.2018.1.29*LG-1137568-2.1.F.'
}

pro_data = requests.get(url_profile,headers=headers)
soup = BeautifulSoup(pro_data.text,'lxml')
print(soup)
'''